/// ===========================================================================
/// THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
/// KIND, WHETHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
/// IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
/// PURPOSE.
/// ===========================================================================
/// 
/// Project:        MOSS Faceted Search
/// Author:         Leonid Lyublinski (leonidly@microsoft.com)
/// Company:        Microsoft Services
/// Date:           05/04/2007  Version:        1.0
///
/// ===========================================================================

using System;
using System.IO;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel;
using System.Text;
using System.Web;
using System.Web.UI;
using System.Web.UI.HtmlControls;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Xml;
using System.Xml.XPath;
using System.Xml.Xsl;
using System.Xml.Serialization;
using System.Data;
using System.Threading;
using Microsoft.SharePoint.WebPartPages;
using Microsoft.Office.Server;
using Microsoft.Office.Server.Search.Administration;
using Microsoft.Office.Server.Search.Query;
using Microsoft.SharePoint;
using Microsoft.SharePoint.WebControls;
using EntLib = Microsoft.Practices.EnterpriseLibrary;
using Microsoft.SharePoint.Portal.ExtendedSearch.Common;

namespace Microsoft.SharePoint.Portal.ExtendedSearch.WebControls
{
    [XmlRoot(Namespace = "Microsoft.SharePoint.Portal.ExtendedSearch")]
    public class SearchProcessor
    {
        #region Fields
        Page _page;
        StringCollection _columns;
        SearchQuery _query;
        bool _moreResultsThanFacets;
        bool _run2ndTime;
        bool _running2ndTime;
        bool _prefixEnabled;
        int _processedTotal;
        bool _cachingEnabled;
        string _hiddenConstraints;
        
        Microsoft.Office.Server.Search.WebControls.CoreResultsWebPart _coreResultsWebPart;
        #endregion

        #region Constants
        internal const string key = "FacetedSearch";
        #endregion

        #region CTOR
        public SearchProcessor(bool cachingEnabled, Page page, StringCollection columns, SearchQuery query, bool prefixEnabled, bool moreResultsThanFacets, bool run2ndTime, bool running2ndTime)
        {
            _cachingEnabled = cachingEnabled;
            _page = page;
            _columns = columns;
            _query = query;
            _prefixEnabled = prefixEnabled;
            _run2ndTime = run2ndTime;
            _running2ndTime = running2ndTime;
            _moreResultsThanFacets = moreResultsThanFacets;
        }
        public SearchProcessor(bool cachingEnabled, Page page, StringCollection columns, SearchQuery query, bool prefixEnabled, bool moreResultsThanFacets, bool run2ndTime, bool running2ndTime, string hiddenConstraints): this(cachingEnabled, page,columns, query, prefixEnabled, moreResultsThanFacets, run2ndTime, running2ndTime)
        {
            _hiddenConstraints = hiddenConstraints;
        }
        #endregion

        /// <summary>
        /// Executes FullTextSqlQuery with constructed earlier SQL statement and returns
        /// DataTable with all Relevant results.
        /// </summary>
        /// <returns>DataTable</returns><permission cref="System.Security.Permissions.RegistryPermission"></permission>
        private DataTable GetResults(int resultsPerPage)
        {            
            _coreResultsWebPart = WPManager.CoreResultsWebPart(_page);
            bool useKeyWordQuery = false;

            ResultTableCollection rtc = null;

            if (_query.QueryMethod == QueryMethodEnum.FullTextSqlQuery)
            {
                useKeyWordQuery = false;
            }
            else if (_query.QueryMethod == QueryMethodEnum.KeywordQuery)
            {
                useKeyWordQuery = true;
            }

            if (useKeyWordQuery)
            {
                rtc = ExecuteGenericQuery<KeywordQuery>(resultsPerPage);
            }
            else
            {
                rtc = ExecuteGenericQuery<FullTextSqlQuery>(resultsPerPage);
            }

            DataTable dt = new DataTable();
            if ((rtc != null) && rtc.Exists(ResultType.RelevantResults))
            {
                using (ResultTable rt = rtc[ResultType.RelevantResults])
                {
                    if (rt.RowCount > 0)
                    {
                        dt.TableName = rt.ResultType.ToString();
                        dt.Load(rt, LoadOption.OverwriteChanges);
                        
                        _moreResultsThanFacets = rt.TotalRows > Math.Min(Constants.RESULTS_PER_PAGE_MIN, resultsPerPage);
                        //_moreResultsThanFacets = rt.TotalRows > Constants.RESULTS_PER_PAGE_MAX;   // responsible for 300 when results <500 and >300. stillneed to review what happens if total results < 300?

                        dt.ExtendedProperties.Add("MoreResultsThanFacets", _moreResultsThanFacets);
                        dt.ExtendedProperties.Add("TotalRows", rt.TotalRows);
                        dt.ExtendedProperties.Add("FieldCount", rt.FieldCount);
                        dt.ExtendedProperties.Add("RowCount", rt.RowCount);
                        dt.ExtendedProperties.Add("IsTotalRowsExact", rt.IsTotalRowsExact);
                        dt.ExtendedProperties.Add("Created", DateTime.Now);
                    }
                }
            }
            return dt;
        }

        /// <summary>
        /// Handles all diffferences between FullTextQuery and KeywordQuery
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void CustomizeQuery(object sender, EventArgs e)
        {
            if (sender is KeywordQuery)
            {
                KeywordQuery query = (KeywordQuery)sender;
                query.QueryText = Utility.BuildQueryText(_query);
                if (string.IsNullOrEmpty(query.QueryText))
                    throw new ArgumentNullException("QueryText", Resource.ERROR_QUERY_EMPTY);
                query.SelectProperties.Add("Path");

                foreach (string c in _columns)
                {
                    query.SelectProperties.Add(c);
                }
                query.HiddenConstraints = _hiddenConstraints;
            }
            if (sender is FullTextSqlQuery)
            {
                FullTextSqlQuery query = (FullTextSqlQuery)sender;
                SqlBuilder sqlBuilder = new SqlBuilder(_query, _columns, _prefixEnabled);
                string _sql = sqlBuilder.SqlQuery;
                query.QueryText = _sql;
                if (!query.QueryText.Contains(" WHERE "))
                    throw new ArgumentNullException("QueryText", Resource.ERROR_QUERY_EMPTY);
                // add logic to handle hidden constraints
            }
        }

        private ResultTableCollection ExecuteGenericQuery<T>(int resultsPerPage) where T : Query
        {
            using (GenericQuery<T> query = new GenericQuery<T>(CustomizeQuery, _coreResultsWebPart))
            {
                query.Query.Hint = QueryHint.OptimizeWithFullTextIndex | QueryHint.OptimizeWithPropertyStore;
                query.Query.KeywordInclusion = KeywordInclusion.AllKeywords; 
                query.Query.RowLimit = resultsPerPage;
                query.Query.StartRow = 0;
                query.Query.SiteContext = new Uri(SPControl.GetContextSite(HttpContext.Current).Url);
                query.Query.AuthenticationType = QueryAuthenticationType.NtAuthenticatedQuery;

                if (SPSecurity.AuthenticationMode != System.Web.Configuration.AuthenticationMode.Windows)
                {
                    query.Query.AuthenticationType = QueryAuthenticationType.PluggableAuthenticatedQuery;
                }

                query.Query.ResultTypes |= ResultType.RelevantResults;

                query.Query.SiteContext = new Uri(GetSiteUrl());
                //execute the query using the query object
                return query.Execute(EventArgs.Empty);
            }
        }

        #region Cache
        private int GetSaveHashCode(object o)
        {
            if (o == null) return 0;
            else if (o is NameValueCollection)
            {
                // Hashcode returned for the NameValueCollection (nor Hashtable) is not unique. Redo via string
                if (((NameValueCollection)o).Count == 0) return 0;
                else
                {
                    NameValueCollection col = (NameValueCollection)o;
                    return Utility.GetHashCode(col);
                }
            }
            else return o.GetHashCode();
        }
        private string LongCacheKey(string selectColumns, short resultsPerPage)
        {
            // key depends on the user (security trimming), selected columns, # of results, keyword, scope
            string key = string.Format("{0}_{1}_{2}_{3}_{4}", _page.User.Identity.Name.GetHashCode(), GetSaveHashCode(selectColumns), resultsPerPage, GetSaveHashCode(_query.Keywords), GetSaveHashCode(_query.Scope));
            return key;
        }
        private string FastCacheKey(string selectColumns, short resultsPerPage)
        {
            // key depends on the user (security trimming), selected columns, # of results, keyword, scope, properties
            string key = string.Format("{0}_{1}_{2}_{3}_{4}_{5}", _page.User.Identity.Name.GetHashCode(), GetSaveHashCode(selectColumns), resultsPerPage, GetSaveHashCode(_query.Keywords), GetSaveHashCode(_query.Scope), GetSaveHashCode(_query.Properties));
            return key;
        }
        private string UserKey()
        {
            // key depends on the user 
            string key = _page.User.Identity.Name.GetHashCode().ToString();
            return key;
        }

        protected internal DataTable GetCachedData(string selectColumns, short resultsPerPage, int longCacheTimeout, int fastCacheTimeout, out bool usingCachedData, out bool run2ndTime)
        {
            string longCacheKey = LongCacheKey(selectColumns, resultsPerPage);
            string fastCacheKey = FastCacheKey(selectColumns, resultsPerPage);
            string userKey = UserKey();
            string userKeyLongCache = string.Format("{0}_{1}", userKey, "long");
            string userKeyFastCache = string.Format("{0}_{1}", userKey, "fast");
            usingCachedData = false;
            run2ndTime = _run2ndTime;
            DataTable dtResult;

            try
            {
                if (!_cachingEnabled)
                {
                    short resultsPerPageActive = (resultsPerPage < Constants.RESULTS_PER_PAGE_MAX) ? resultsPerPage : Constants.RESULTS_PER_PAGE_MIN;
                    dtResult = (_running2ndTime) ? GetResults(resultsPerPage) : GetResults(resultsPerPageActive);
                    usingCachedData = false;
                    int totalRows = Convert.ToInt32(dtResult.ExtendedProperties["TotalRows"]);
                    int currentRows = dtResult.Rows.Count;
                    if (_running2ndTime) 
                        run2ndTime = false;
                    else
                        run2ndTime = Run2ndTime(currentRows, totalRows, resultsPerPage);
                }
                else
                {
                    EntLib.Caching.CacheManager cache = EntLib.Caching.CacheFactory.GetCacheManager();

                    // I. fast Cache. Only 1st run
                    // check if cache exist, then read it
                    if (cache.Contains(userKeyLongCache) && cache[userKeyLongCache].ToString() == longCacheKey)
                    {
                        DataTable dtCached = (DataTable)cache[longCacheKey];
                        usingCachedData = true;
                        dtResult = FilteredData(dtCached);
                        int totalRows = Convert.ToInt32(dtResult.ExtendedProperties["TotalRows"]);
                        bool isRefinement = _query.Properties != null && _query.Properties.HasKeys();
                        run2ndTime = (totalRows > resultsPerPage && isRefinement);
                    }
                    else
                    {
                        if (!_running2ndTime)
                        {
                            if (cache.Contains(userKeyFastCache) && cache[userKeyFastCache].ToString() == fastCacheKey)
                            {
                                dtResult = (DataTable)cache[fastCacheKey];
                                usingCachedData = true;
                            }
                            else
                            {
                                short resultsPerPageActive = (resultsPerPage < Constants.RESULTS_PER_PAGE_MAX) ? resultsPerPage : Constants.RESULTS_PER_PAGE_MIN;
                                dtResult = GetResults(resultsPerPageActive);
                                int totalRows = Convert.ToInt32(dtResult.ExtendedProperties["TotalRows"]);
                                int currentRows = dtResult.Rows.Count;
                                run2ndTime = Run2ndTime(currentRows, totalRows, resultsPerPage);
                                cache.Add(userKeyFastCache, fastCacheKey, EntLib.Caching.CacheItemPriority.Normal, null, new EntLib.Caching.Expirations.AbsoluteTime(TimeSpan.FromMinutes(fastCacheTimeout)));
                                cache.Add(fastCacheKey, dtResult, EntLib.Caching.CacheItemPriority.Normal, null, new EntLib.Caching.Expirations.AbsoluteTime(TimeSpan.FromMinutes(fastCacheTimeout)));
                            }
                        }
                        else
                        {
                            if (_query.Properties == null || !_query.Properties.HasKeys())
                            {
                                // clean previous cached key
                                if (cache.Contains(userKeyLongCache))
                                {
                                    string prevLongCacheKey = cache[userKeyLongCache].ToString();
                                    cache.Remove(prevLongCacheKey);
                                }
                                dtResult = GetResults(resultsPerPage);
                                if (dtResult == null) return null;
                                // should check the coverage before caching? or rely on refresh?
                                cache.Add(userKeyLongCache, longCacheKey, EntLib.Caching.CacheItemPriority.Normal, null, new EntLib.Caching.Expirations.AbsoluteTime(TimeSpan.FromMinutes(longCacheTimeout)));
                                cache.Add(longCacheKey, dtResult, EntLib.Caching.CacheItemPriority.Normal, null, new EntLib.Caching.Expirations.AbsoluteTime(TimeSpan.FromMinutes(longCacheTimeout)));
                            }
                            else
                            {
                                dtResult = GetResults(resultsPerPage);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                dtResult = (_running2ndTime) ? GetResults(resultsPerPage) : GetResults(Constants.RESULTS_PER_PAGE_MIN);
                usingCachedData = false;
                Utility.HandleException(ex, false, null, null);
            }
            //TestFacets(dtResult); //- uncomment when need to see the source data

            return dtResult;
        }
        protected internal void ReleaseCache(out bool usingCachedData)
        {
            EntLib.Caching.CacheManager cache = EntLib.Caching.CacheFactory.GetCacheManager();
            string userKey = UserKey();
            string userKeyLongCache = string.Format("{0}_{1}", userKey, "long");
            string userKeyFastCache = string.Format("{0}_{1}", userKey, "fast");
            string longCacheKey = cache[userKeyLongCache] as string;
            string fastCacheKey = cache[userKeyFastCache] as string;
            if (userKeyFastCache != null) cache.Remove(userKeyFastCache);
            if (fastCacheKey != null) cache.Remove(fastCacheKey);
            if (userKeyLongCache != null) cache.Remove(userKeyLongCache);
            if (longCacheKey != null) cache.Remove(longCacheKey);
            usingCachedData = false;
        }
        #endregion

        /// <summary>
        /// A test method used to display Grid with search results for testing
        /// </summary>
        /// <param name="dt"></param>
        private void TestFacets(DataTable dt)
        {
            GridView view = new GridView();
            view.DataSource = dt;
            view.DataBind();
            view.GridLines = GridLines.Both;
            _page.Controls.Add(view);
        }

        /// <summary>
        /// Performant way to filter datatable
        /// </summary>
        /// <param name="dt">DataTable</param>
        /// <param name="properties">NameValueCollection</param>
        /// <returns>DataTable</returns>
        private DataTable FilteredData(DataTable dt)
        {            
            NameValueCollection properties = _query.Properties;
            if (properties.Count > 0)
            {
                dt.CaseSensitive = false;
                string sort = string.Join(", ", properties.AllKeys);
                ArrayList values = new ArrayList(properties);
                for (int i = 0; i < properties.AllKeys.Length; i++)
                {
                    values[i] = properties[properties.AllKeys.GetValue(i).ToString()];
                }
                string[] filter = (string[])values.ToArray(typeof(string));
                DataView view = new DataView(dt, string.Empty, sort, DataViewRowState.CurrentRows);
                DataTable dtFiltered = dt.Clone();
                DataRowView[] rows = view.FindRows(filter);
                foreach (DataRowView drv in rows)
                {
                    dtFiltered.Rows.Add(drv.Row.ItemArray);
                }
                return dtFiltered;

            }
            else return dt;                     
        }

        /// <summary>
        /// Returns the active url
        /// </summary>
        /// <returns></returns>
        private string GetSiteUrl()
        {
            String url = String.Empty;
            
            HttpContext currentContext = HttpContext.Current;

            if (currentContext == null)
                return url;
            SPWeb web = SPControl.GetContextWeb(HttpContext.Current);

            if (web == null)
                return url;

            url = web.Url;
            if (String.IsNullOrEmpty(url))
                return String.Empty;

            if (url.EndsWith("/"))
                return url;

            return url + "/";
        }

        /// <summary>
        /// This method builds logic to run the 2nd thread
        /// </summary>
        /// <param name="currentRows"></param>
        /// <param name="totalRows"></param>
        /// <param name="resultsPerPage"></param>
        /// <returns></returns>
        private bool Run2ndTime(int currentRows, int totalRows, int resultsPerPage)
        {
            // when to run 2nd thread? 
            // 1) there is more results than returned by 1st thread
            // 2) 1st thread returned less results that asked
            // 3) only if still running 1st thread
            // 4) more results asked than Constants.RESULTS_PER_PAGE_MAX.                     
            return (currentRows < resultsPerPage) && (currentRows < totalRows) && (resultsPerPage > Constants.RESULTS_PER_PAGE_MAX);
                
        }
    }
}
